{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import os\n",
    "import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### ***Data Generation***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trn_path='/data/DW/DataFusion/dfc2019-master/track4/log_data_folder/model_fusion/test_trn_fea/'\n",
    "val_path='/data/DW/DataFusion/dfc2019-master/track4/log_data_folder/model_fusion/test_val_fea/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_list_of_files(path):\n",
    "    \n",
    "    trn_files = glob.glob(os.path.join(path,\"*_FEA.npy\"))\n",
    "    trn_labs= glob.glob(os.path.join(path,\"*_FEA_LAB.npy\"))\n",
    "    return sorted(trn_files),sorted(trn_labs)\n",
    "\n",
    "trn_files,trn_labs=get_list_of_files(trn_path)\n",
    "val_files,val_labs=get_list_of_files(val_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "observation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/data/DW/DataFusion/dfc2019-master/track4/log_data_folder/model_fusion/test_trn_fea/JAX_004_FEA.npy\n",
      "/data/DW/DataFusion/dfc2019-master/track4/log_data_folder/model_fusion/test_trn_fea/JAX_004_FEA_LAB.npy\n",
      "/data/DW/DataFusion/dfc2019-master/track4/log_data_folder/model_fusion/test_val_fea/JAX_260_FEA.npy\n",
      "/data/DW/DataFusion/dfc2019-master/track4/log_data_folder/model_fusion/test_val_fea/JAX_260_FEA_LAB.npy\n",
      "(155648, 128)\n",
      "(155648, 1)\n",
      "(155648, 5)\n"
     ]
    }
   ],
   "source": [
    "print(trn_files[0])\n",
    "print(trn_labs[0])\n",
    "print(val_files[0])\n",
    "print(val_labs[0])\n",
    "print(np.load(trn_files[0]).shape)\n",
    "print(np.load(trn_labs[0]).shape)\n",
    "print(np.load(trn_files[0][:-8]+'_XYZIR.npy').shape)##coodinate file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/data/DW/DataFusion/dfc2019-master/track4/log_data_folder/model_fusion/test_val_fea/JAX_260\n"
     ]
    }
   ],
   "source": [
    "print(val_files[0][:-8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.83554584 0.8009994  0.809932   ... 0.33603504 0.         0.4673449 ]\n",
      " [0.         0.08614694 0.         ... 0.         0.         0.        ]\n",
      " [0.56692135 1.211102   0.62345016 ... 0.76880795 0.         0.78496736]\n",
      " ...\n",
      " [0.22486682 1.0475065  0.20512994 ... 0.48153156 0.         0.35670096]\n",
      " [0.         0.         0.         ... 0.         0.6968867  0.90659523]\n",
      " [1.2816851  1.7837983  1.439342   ... 1.0023284  0.         1.3156325 ]]\n"
     ]
    }
   ],
   "source": [
    "print(np.load(trn_files[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 2 5 6 9]\n"
     ]
    }
   ],
   "source": [
    "print(np.unique(np.load(trn_labs[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4/4 [00:02<00:00,  1.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(622592, 128)\n",
      "(622592, 1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "for i in tqdm(range(len(trn_files))):\n",
    "    temp_trndata=np.load(trn_files[i])\n",
    "    temp_trnlab=np.load(trn_labs[i])\n",
    "    if i==0:\n",
    "        X_trn=temp_trndata\n",
    "        y_trn=temp_trnlab\n",
    "    else:\n",
    "        X_trn=np.concatenate((X_trn,temp_trndata),axis=0)\n",
    "        y_trn=np.concatenate((y_trn,temp_trnlab),axis=0)\n",
    "\n",
    "print(X_trn.shape)\n",
    "print(y_trn.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_trn=np.squeeze(y_trn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### las 2 sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_trn_seq=y_trn\n",
    "y_trn_seq[y_trn_seq==0]=255\n",
    "y_trn_seq[y_trn_seq==2]=0\n",
    "y_trn_seq[y_trn_seq==5]=1\n",
    "y_trn_seq[y_trn_seq==6]=2\n",
    "y_trn_seq[y_trn_seq==9]=3\n",
    "y_trn_seq[y_trn_seq==17]=4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### clean the sample that $\\quad \\text{y_{trn_seq}}==255$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trn_idx=y_trn_seq<255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_trn_ult=X_trn[trn_idx]\n",
    "y_trn_ult=y_trn_seq[trn_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(587746, 128) (587746,)\n"
     ]
    }
   ],
   "source": [
    "print(X_trn_ult.shape,y_trn_ult.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, 3, 4], dtype=uint8)"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(y_trn_ult)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "random choosing samples to trn shorter time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 128) (10000,)\n"
     ]
    }
   ],
   "source": [
    "idx=np.random.choice(X_trn_ult.shape[0],10000,replace=False)\n",
    "\n",
    "X_trn_new=X_trn_ult[idx]\n",
    "y_trn_new=y_trn_ult[idx]\n",
    "\n",
    "print(X_trn_new.shape,y_trn_new.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### sequence 2 las"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "LAB_MAP={0:2,1:5,2:6,3:9,4:17}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### ***SVM***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Gridsearch CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [],
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "grid_SVM=GridSearchCV(SVC(decision_function_shape='ovo'), param_grid={'kernel':['linear'],\n",
    "                        'C': [1]},cv=5)\n",
    "grid_SVM.fit(X_trn,y_trn)\n",
    "print(\"Best C is %f, Best gamma is %f\" %(grid_SVM.best_params_['C'],grid_SVM.best_params_['gamma']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for mu_SVM,std_SVM,param_SVM in zip(grid_SVM.cv_results_['mean_test_score'],\n",
    "                                   grid_SVM.cv_results_['std_test_score'],\n",
    "                                   grid_SVM.cv_results_['params']):\n",
    "    print('Mean test error: %.3f(+/-%.3f) with %r'%(mu_SVM,std_SVM,param_SVM))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train & Save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.externals import joblib\n",
    "\n",
    "def trn_SVM(X_trn,y_trn,C):\n",
    "    SVM_model=SVC(C=C,kernel='linear',decision_function_shape='ovo')\n",
    "    SVM_model.fit(X_trn,y_trn)\n",
    "    joblib.dump(SVM_model,'/data/DW/DataFusion/dfc2019-master/track4/log_data_folder/model_fusion/test_trn_fea/SVM_model.m')\n",
    "    return SVM_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training SVM model cost 2.361832857131958 s\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "time_start=time.time()\n",
    "SVM_model=trn_SVM(X_trn_new,y_trn_new,1)\n",
    "time_end=time.time()\n",
    "print('training SVM model cost',time_end-time_start,'s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OA: 0.08571624755859375\n",
      "OA: 0.05738525390625\n",
      "SVM model predicting cost 77.61838841438293 s\n"
     ]
    }
   ],
   "source": [
    "time_start=time.time()\n",
    "for i in range(len(val_files)):\n",
    "    X_val=np.load(val_files[i])\n",
    "    y_val=np.load(val_labs[i])\n",
    "    y_pre_val_SVM=[]\n",
    "    y_pred=SVM_model.predict(X_val)\n",
    "    y_pre_val_SVM=np.array([LAB_MAP[x] for x in y_pred])\n",
    "    #np.save(val_files[i][:-8]+'_SVM_LAB.npy',y_pre_val_SVM)\n",
    "    print('OA:',np.mean(y_pre_val_SVM==y_val[:,0]))\n",
    "time_end=time.time()\n",
    "print('SVM model predicting cost',time_end-time_start,'s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2,  5,  6,  9, 17])"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(y_pre_val_SVM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SVM_model=joblib.load('/data/DW/DataFusion/dfc2019-master/track4/log_data_folder/model_fusion/test_trn_fea/SVM_model.m')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred=SVM_model.predict(X_val)\n",
    "y_pre_val_SVM2=np.array([LAB_MAP[x] for x in y_pred])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OA: 0.90489501953125\n"
     ]
    }
   ],
   "source": [
    "print('OA:',np.mean(y_pre_val_SVM2==y_val[:,0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(163840, 128) (163840, 1)\n"
     ]
    }
   ],
   "source": [
    "print(X_val.shape,y_val.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### ***RF***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "grid_RF=GridSearchCV(RandomForestClassifier(), \n",
    "                     param_grid={'n_estimators': [10,20,30,40,50,60,70,80,90,100]},cv=5)\n",
    "grid_RF.fit(X_trn,y_trn)\n",
    "print(\"Best number of estimators is %f\" %(grid_RF.best_params_['n_estimators']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def trn_RF(X_trn,y_trn,n_esti):\n",
    "    RF_model=RandomForestClassifier(oob_score=True,n_estimators=n_esti,random_state=None,max_features='sqrt')#sqrt(c)\n",
    "    RF_model.fit(X_trn,y_trn)\n",
    "    joblib.dump(RF_model,'/data/DW/DataFusion/dfc2019-master/track4/log_data_folder/model_fusion/test_trn_fea/RF_model.m')\n",
    "    return RF_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training RF model cost 7.401139259338379 s\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "time_start=time.time()\n",
    "RF_model=trn_RF(X_trn_new,y_trn_new,100)\n",
    "time_end=time.time()\n",
    "print('training RF model cost',time_end-time_start,'s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OA: 0.2556835612631403\n",
      "OA: 0.3753878366947174\n",
      "RF model predicting cost 195.84533166885376 s\n"
     ]
    }
   ],
   "source": [
    "time_start=time.time()\n",
    "for i in range(len(val_files)):\n",
    "    X_val=np.load(val_files[i])\n",
    "    y_val=np.load(val_labs[i])\n",
    "    y_pre_val_RF=[]\n",
    "    y_pred=RF_model.predict(X_val)\n",
    "    y_pre_val_RF=np.array([LAB_MAP[x] for x in y_pred])\n",
    "    #np.save(os.path.join(path,\"_RF_LAB.npy\"),y_pre_val_RF)\n",
    "    print('OA:',np.mean(y_pre_val_RF==y_val))\n",
    "time_end=time.time()\n",
    "print('RF model predicting cost',time_end-time_start,'s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2,  5,  6,  9, 17])"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(y_pre_val_RF)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### ***RoF***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from rotation_forest import RotationTreeClassifier, RotationForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "grid_RoF=GridSearchCV(RotationForestClassifier(), \n",
    "                     param_grid={'n_estimators': [10,20,30,40,50,60,70,80,90,100]},cv=5)\n",
    "grid_RoF.fit(X_trn,y_trn)\n",
    "print(\"Best number of estimators is %f\" %(grid_RF.best_params_['n_estimators']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def trn_RoF(X_trn,y_trn):\n",
    "    sel_feat=int(np.sqrt(X_trn.shape[1]))\n",
    "    RoF_model=RotationForestClassifier(n_estimators=20,random_state=None,max_features=sel_feat)#sqrt(c)\n",
    "    RoF_model.fit(X_trn,y_trn)\n",
    "    joblib.dump(RoF_model,'/data/DW/DataFusion/dfc2019-master/track4/log_data_folder/model_fusion/test_trn_fea/RoF_model.m')\n",
    "    return RoF_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training RoF model cost 33.654308557510376 s\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "time_start=time.time()\n",
    "RoF_model=trn_RoF(X_trn_new,y_trn_new)\n",
    "time_end=time.time()\n",
    "print('training RoF model cost',time_end-time_start,'s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OA: 0.25628676800988615\n",
      "OA: 0.3749700081720948\n",
      "RoF model predicting cost 192.50943231582642 s\n"
     ]
    }
   ],
   "source": [
    "time_start=time.time()\n",
    "for i in range(len(val_files)):\n",
    "    X_val=np.load(val_files[i])\n",
    "    y_val=np.load(val_labs[i])\n",
    "    y_pre_val_RoF=[]\n",
    "    y_pred=RoF_model.predict(X_val)\n",
    "    y_pre_val_RoF=np.array([LAB_MAP[x] for x in y_pred])\n",
    "    #np.save(os.path.join(path,\"_RF_LAB.npy\"),y_pre_val_RF)\n",
    "    print('OA:',np.mean(y_pre_val_RoF==y_val))\n",
    "time_end=time.time()\n",
    "print('RoF model predicting cost',time_end-time_start,'s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### ***AB***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "grid_AB=GridSearchCV(AdaBoostClassifier(), \n",
    "                     param_grid={'n_estimators': [10,20,30,40,50,60,70,80,90,100]},cv=5)\n",
    "grid_AB.fit(X_trn,y_trn)\n",
    "print(\"Best number of estimators is %f\" %(grid_AB.best_params_['n_estimators']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def trn_AB(X_trn,y_trn):\n",
    "    AB_model=AdaBoostClassifier(n_estimators=100,random_state=None)#sqrt(c)\n",
    "    AB_model.fit(X_trn,y_trn)\n",
    "    joblib.dump(AB_model,'/data/DW/DataFusion/dfc2019-master/track4/log_data_folder/model_fusion/test_trn_fea/AB_model.m')\n",
    "    return AB_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training AB model cost 15.588809490203857 s\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "time_start=time.time()\n",
    "AB_model=trn_AB(X_trn_new,y_trn_new)\n",
    "time_end=time.time()\n",
    "print('training AB model cost',time_end-time_start,'s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OA: 0.2869176013045944\n",
      "OA: 0.3730746987834573\n",
      "AB model predicting cost 207.84754490852356 s\n"
     ]
    }
   ],
   "source": [
    "time_start=time.time()\n",
    "for i in range(len(val_files)):\n",
    "    X_val=np.load(val_files[i])\n",
    "    y_val=np.load(val_labs[i])\n",
    "    y_pre_val_AB=[]\n",
    "    y_pred=AB_model.predict(X_val)\n",
    "    y_pre_val_AB=np.array([LAB_MAP[x] for x in y_pred])\n",
    "    #np.save(os.path.join(path,\"_RF_LAB.npy\"),y_pre_val_RF)\n",
    "    print('OA:',np.mean(y_pre_val_AB==y_val))\n",
    "time_end=time.time()\n",
    "print('AB model predicting cost',time_end-time_start,'s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  2,  5,  6,  9, 17], dtype=uint8)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 5, 6])"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(y_pre_val_AB)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### ***GB***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-46-59d4e45c69da>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m grid_GB=GridSearchCV(GradientBoostingClassifier(), \n\u001b[1;32m      2\u001b[0m                      param_grid={'n_estimators': [10,20,30,40,50,60,70,80,90,100],'learning_rate':[0.001,0.01,0.1]},cv=5)\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mgrid_GB\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_trn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_trn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Best number of estimators is %f\"\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrid_GB\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_params_\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'n_estimators'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, groups, **fit_params)\u001b[0m\n\u001b[1;32m    720\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresults_container\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    721\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 722\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    723\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    724\u001b[0m         \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresults_container\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36m_run_search\u001b[0;34m(self, evaluate_candidates)\u001b[0m\n\u001b[1;32m   1189\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1190\u001b[0m         \u001b[0;34m\"\"\"Search all candidates in param_grid\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1191\u001b[0;31m         \u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mParameterGrid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1192\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1193\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mevaluate_candidates\u001b[0;34m(candidate_params)\u001b[0m\n\u001b[1;32m    709\u001b[0m                                \u001b[0;32mfor\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    710\u001b[0m                                in product(candidate_params,\n\u001b[0;32m--> 711\u001b[0;31m                                           cv.split(X, y, groups)))\n\u001b[0m\u001b[1;32m    712\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    713\u001b[0m                 \u001b[0mall_candidate_params\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcandidate_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m    981\u001b[0m             \u001b[0;31m# remaining jobs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    982\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 983\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    984\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_iterator\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    985\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[0;34m(self, iterator)\u001b[0m\n\u001b[1;32m    823\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    824\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 825\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    826\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    827\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    780\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    781\u001b[0m             \u001b[0mjob_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 782\u001b[0;31m             \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    783\u001b[0m             \u001b[0;31m# A job can complete so quickly than its callback is\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    784\u001b[0m             \u001b[0;31m# called before we get here, causing self._jobs to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/sklearn/externals/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[0;34m(self, func, callback)\u001b[0m\n\u001b[1;32m    180\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    181\u001b[0m         \u001b[0;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 182\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    183\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/sklearn/externals/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    543\u001b[0m         \u001b[0;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    544\u001b[0m         \u001b[0;31m# arguments in memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 545\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    546\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    547\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    259\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 261\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    259\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 261\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36m_fit_and_score\u001b[0;34m(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, return_estimator, error_score)\u001b[0m\n\u001b[1;32m    526\u001b[0m             \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    527\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 528\u001b[0;31m             \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    529\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    530\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/sklearn/ensemble/gradient_boosting.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight, monitor)\u001b[0m\n\u001b[1;32m   1457\u001b[0m         \u001b[0mX_idx_sorted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1458\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mpresort\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1459\u001b[0;31m             X_idx_sorted = np.asfortranarray(np.argsort(X, axis=0),\n\u001b[0m\u001b[1;32m   1460\u001b[0m                                              dtype=np.int32)\n\u001b[1;32m   1461\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36margsort\u001b[0;34m(a, axis, kind, order)\u001b[0m\n\u001b[1;32m    971\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    972\u001b[0m     \"\"\"\n\u001b[0;32m--> 973\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_wrapfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'argsort'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkind\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkind\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    974\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    975\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.5/site-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36m_wrapfunc\u001b[0;34m(obj, method, *args, **kwds)\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_wrapfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[0;31m# An AttributeError occurs if the object does not have\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "grid_GB=GridSearchCV(GradientBoostingClassifier(), \n",
    "                     param_grid={'n_estimators': [10,20,30,40,50,60,70,80,90,100],'learning_rate':[0.001,0.01,0.1]},cv=5)\n",
    "grid_GB.fit(X_trn,y_trn)\n",
    "print(\"Best number of estimators is %f\" %(grid_GB.best_params_['n_estimators']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def trn_GB(X_trn,y_trn):\n",
    "    GB_model=GradientBoostingClassifier(n_estimators=100,learning_rate=0.1,random_state=None)#sqrt(c)\n",
    "    GB_model.fit(X_trn,y_trn)\n",
    "    joblib.dump(GB_model,'/data/DW/DataFusion/dfc2019-master/track4/log_data_folder/model_fusion/test_trn_fea/GB_model.m')\n",
    "    return GB_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training GB model cost 85.37349510192871 s\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "time_start=time.time()\n",
    "GB_model=trn_GB(X_trn_new,y_trn_new)\n",
    "time_end=time.time()\n",
    "print('training GB model cost',time_end-time_start,'s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OA: 0.27783634938532487\n",
      "OA: 0.3759371126443148\n",
      "GB model predicting cost 198.82611322402954 s\n"
     ]
    }
   ],
   "source": [
    "time_start=time.time()\n",
    "for i in range(len(val_files)):\n",
    "    X_val=np.load(val_files[i])\n",
    "    y_val=np.load(val_labs[i])\n",
    "    y_pre_val_GB=[]\n",
    "    y_pred=GB_model.predict(X_val)\n",
    "    y_pre_val_GB=np.array([LAB_MAP[x] for x in y_pred])\n",
    "    #np.save(os.path.join(path,\"_RF_LAB.npy\"),y_pre_val_RF)\n",
    "    print('OA:',np.mean(y_pre_val_GB==y_val))\n",
    "time_end=time.time()\n",
    "print('GB model predicting cost',time_end-time_start,'s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### ***LGB***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_data=lgb.Dataset(X_trn_new,label=y_trn_new)\n",
    "params={\n",
    "    'learning_rate':0.01,\n",
    "    'max_depth':1,\n",
    "    'application':'multiclass',\n",
    "    'num_class':5,\n",
    "    'verbose':-1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training LGB model cost 232.12101793289185 s\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "time_start=time.time()\n",
    "LGB_model=lgb.train(params,train_data)\n",
    "time_end=time.time()\n",
    "print('training LGB model cost',time_end-time_start,'s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OA: 0.28628260194091126\n",
      "OA: 0.38213308673352003\n",
      "LGB model predicting cost 189.33775854110718 s\n"
     ]
    }
   ],
   "source": [
    "time_start=time.time()\n",
    "for i in range(len(val_files)):\n",
    "    X_val=np.load(val_files[i])\n",
    "    y_val=np.load(val_labs[i])\n",
    "    y_pre_val_LGB=[]\n",
    "    y_pred=LGB_model.predict(X_val)#one-hot\n",
    "    y_pred_seq=np.argmax(y_pred,axis=1)\n",
    "    y_pre_val_LGB=np.array([LAB_MAP[x] for x in y_pred_seq])\n",
    "    #np.save(os.path.join(path,\"_RF_LAB.npy\"),y_pre_val_RF)\n",
    "    print('OA:',np.mean(y_pre_val_LGB==y_val))\n",
    "time_end=time.time()\n",
    "print('LGB model predicting cost',time_end-time_start,'s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### ***XGB***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "xgb_traindata = xgb.DMatrix(X_trn_new, label=y_trn_new) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##https://blog.csdn.net/kwame211/article/details/81098025 \n",
    "params = {\n",
    "    'booster': 'gbtree',\n",
    "    'objective': 'multi:softmax',  # 多分类的问题\n",
    "    'num_class': 5,               # 类别数，与 multisoftmax 并用\n",
    "    'gamma': 0.1,                  # 用于控制是否后剪枝的参数,越大越保守，一般0.1、0.2这样子。\n",
    "    'max_depth': 3,               # 构建树的深度，越大越容易过拟合\n",
    "    'lambda': 2,                   # 控制模型复杂度的权重值的L2正则化项参数，参数越大，模型越不容易过拟合。\n",
    "    'subsample': 0.7,              # 随机采样训练样本\n",
    "    'colsample_bytree': 0.7,       # 生成树时进行的列采样\n",
    "    'min_child_weight': 3,\n",
    "    'silent': 1,                   # 设置成1则没有运行信息输出，最好是设置为0.\n",
    "    'eta': 0.007,                  # 如同学习率\n",
    "    'seed': 1000,\n",
    "    'nthread': 4                   # cpu 线程数\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training XGB model cost 0.8037967681884766 s\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "time_start=time.time()\n",
    "num_round = 10\n",
    "XGB_model = xgb.train(params, xgb_traindata, num_round)\n",
    "time_end=time.time()\n",
    "print('training XGB model cost',time_end-time_start,'s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0. 1. 2. 3. 4.]\n",
      "OA: 0.21600613999180496\n",
      "[0. 1. 2. 3. 4.]\n",
      "OA: 0.3756059022992849\n",
      "XGB model predicting cost 179.4974410533905 s\n"
     ]
    }
   ],
   "source": [
    "time_start=time.time()\n",
    "for i in range(len(val_files)):\n",
    "    X_val=np.load(val_files[i])\n",
    "    y_val=np.load(val_labs[i])\n",
    "    y_pre_val_XGB=[]\n",
    "    y_pred=XGB_model.predict(xgb.DMatrix(X_val))#one-hot\n",
    "    print(np.unique(y_pred))\n",
    "    y_pre_val_XGB=np.array([LAB_MAP[x] for x in y_pred])\n",
    "    #np.save(os.path.join(path,\"_RF_LAB.npy\"),y_pre_val_RF)\n",
    "    print('OA:',np.mean(y_pre_val_XGB==y_val))\n",
    "time_end=time.time()\n",
    "print('XGB model predicting cost',time_end-time_start,'s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### ***ANN***"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (py4torch_tf)",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
